program ExploitDB;

include <core.vhs>
include <shell.vhs>
include <netutil.vhs>
include <database.vhs>

import core::*;
import core::Pattern::*;
import core::Logging::*;
import netutil::*;
import database::*;
import database::CSV::*;

const dbname='output/exploitdb.csv';
const dbconf='ID,CVEID,OSVDBID,PublishedDate,Verified,Title';

function insert_sap(db, ...) CSV::insert(db, ...)

function downloadExploitDb(database, startID='1') {
	var outfunc = function(data) {
		database.writeCsvLine([data.id, data.cve, data.osvdb, data.date, data.verified, data.title]);
	}
	var total=0;
	var nextid = startID;
	var downloadExploit=function(id='1', outfunc) {
		log('Download exploit ID: %s', id);
		var url = 'http://www.exploit-db.com/exploits/' + id + '/';
		var f = download(url, timeout=500000, cacheFile='webcache/e' + id + '.html');
		if (f == null) return null; // false to download
		var ip = new SequenceStream(f, 4096);
		var res = ParseExploitDB(input=ip, id=id, outfunc=outfunc);
		ip.close();
		return res;
	}
	log('Start downloading exploitdb');
	while (nextid != null && nextid != '') {
		total++;
		nextid = downloadExploit(nextid, outfunc);
	}
	log('Download completed. %d entries downloaded', total);
	
}

pattern ParseExploitDB(id, outfunc) {
	locate('<div'); // the information table
	locate_end('div', operator=LOCATOR_CLOSE_TAG);
	
	var title = ExploitDB_Header();
	var cve = ifPatternMatchFail(ExploitDB_CVE(),'');
	var osvdb = ifPatternMatchFail(ExploitDB_OSVDB(), '');
	var date = ifPatternMatchFail(ExploitDB_Publish(), '');
	var verified = ExploitDB_Verified();
	
	outfunc?(struct { id=id, cve=cve, osvdb=osvdb, date=date, verified=verified, title=title.trim() } );
	locate('alt="Home"');
	var nextid = ifPatternMatchFail(ExploitDB_NextID(), '');
	return nextid;
}

pattern<DataPart> ExploitDB_Header($dataPattern=@'<h1.+?>(.+?)</h1>') this[1]
pattern<DataPart> ExploitDB_CVE($minOccur=0, $dataPattern=@'CVE:.+?href="http://cve.mitre.org/cgi-bin/cvename.cgi\?name=CVE-(\d+-\d+)"') this[1]
pattern<DataPart> ExploitDB_Publish($minOccur=0, $dataPattern=@'Published:\s+(\d+-\d+-\d+)') this[1]
pattern<DataPart> ExploitDB_OSVDB($minOccur=0, $dataPattern=@'OSVDB-ID:.+?href="http://osvdb.org/show/osvdb/(\d+)"') this[1]
pattern<DataPart> ExploitDB_Verified($minOccur=0, $dataPattern=@'Verified:.+?alt="(.+?)" />') this[1]
pattern<DataPart> ExploitDB_NextID($minOccur=0, $dataPattern=@'http://www.exploit-db.com/exploits/(\d+)/') this[1];

function main() {
	Logging::configureLog("download-exploit.log");
	netutil::setExpiredSecPeriod(SEC_PER_MONTH);
	
	var db=openDatabase(dbname, dbconf);
	downloadExploitDb(database=db, startID='19297');
	closeDatabase(db);
}
